import re
import time
import traceback
import logging
import platform
import os
import requests
import json
import pymysql
import random
import subprocess
from pathlib import Path
from datetime import datetime, timedelta
from playwright.sync_api import sync_playwright, expect, Error as PlaywrightError
from bs4 import BeautifulSoup
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from config import DB_CONFIG, WEIXIN_CONFIG, VIDEO_CONFIG, LOG_CONFIG, BROWSER_CONFIG

# 配置日志
logging.basicConfig(
    level=getattr(logging, LOG_CONFIG["level"]),
    format=LOG_CONFIG["format"]
)
logger = logging.getLogger(__name__)


class BrowserManager:
    """浏览器管理器，负责Firefox浏览器的下载、检查和管理。"""
    
    @staticmethod
    def check_browser_status():
        """检查浏览器状态。"""
        browser_dir = BROWSER_CONFIG['local_browser_dir']
        firefox_path = BROWSER_CONFIG['firefox_path']
        
        status = {
            'browser_dir_exists': browser_dir.exists(),
            'firefox_executable_exists': firefox_path.exists(),
            'browser_dir_path': str(browser_dir),
            'firefox_path': str(firefox_path)
        }
        
        if status['browser_dir_exists']:
            try:
                total_size = sum(f.stat().st_size for f in browser_dir.rglob('*') if f.is_file())
                status['browser_dir_size_mb'] = total_size / (1024 * 1024)
            except Exception:
                status['browser_dir_size_mb'] = 0
        
        return status
    
    @staticmethod
    def download_firefox(force=False):
        """下载Firefox浏览器到项目目录。"""
        firefox_path = BROWSER_CONFIG['firefox_path']
        
        if firefox_path.exists() and not force:
            logger.info(f"Firefox已存在: {firefox_path}")
            return True
        
        logger.info("开始下载Firefox浏览器...")
        
        try:
            browser_dir = BROWSER_CONFIG['local_browser_dir']
            browser_dir.mkdir(parents=True, exist_ok=True)
            
            # 设置环境变量
            env = os.environ.copy()
            env['PLAYWRIGHT_BROWSERS_PATH'] = str(browser_dir)
            
            # 执行下载命令
            result = subprocess.run([
                sys.executable, '-m', 'playwright', 'install', 'firefox'
            ], env=env, capture_output=True, text=True, timeout=300)
            
            if result.returncode == 0:
                logger.info("Firefox下载成功！")
                # 查找并更新Firefox路径
                if BrowserManager._find_firefox_executable():
                    logger.info(f"Firefox可执行文件位于: {BROWSER_CONFIG['firefox_path']}")
                    return True
                else:
                    logger.warning("下载完成但未找到可执行文件")
                    return False
            else:
                logger.error(f"Firefox下载失败: {result.stderr}")
                return False
                
        except subprocess.TimeoutExpired:
            logger.error("下载超时！请检查网络连接。")
            return False
        except Exception as e:
            logger.error(f"下载过程中发生错误: {e}")
            return False
    
    @staticmethod
    def _find_firefox_executable():
        """查找Firefox可执行文件并更新配置。"""
        browser_dir = BROWSER_CONFIG['local_browser_dir']
        
        # 根据操作系统确定文件名
        if platform.system() == "Windows":
            executable_name = "firefox.exe"
            firefox_patterns = [
                "firefox-*/firefox/firefox.exe",
                "firefox/firefox.exe"
            ]
        else:
            executable_name = "firefox"
            firefox_patterns = [
                "firefox-*/firefox/firefox",
                "firefox/firefox"
            ]
        
        # 搜索Firefox可执行文件
        for pattern in firefox_patterns:
            matches = list(browser_dir.glob(pattern))
            if matches:
                firefox_path = matches[0]
                BROWSER_CONFIG['firefox_path'] = firefox_path
                logger.info(f"找到Firefox: {firefox_path}")
                return True
        
        # 如果没找到，进行深度搜索
        for file_path in browser_dir.rglob(executable_name):
            if file_path.is_file():
                BROWSER_CONFIG['firefox_path'] = file_path
                logger.info(f"找到Firefox: {file_path}")
                return True
        
        logger.warning("未找到Firefox可执行文件")
        return False
    
    @staticmethod
    def test_firefox():
        """测试Firefox是否可以正常启动。"""
        firefox_path = BROWSER_CONFIG['firefox_path']
        
        if not firefox_path.exists():
            logger.error("Firefox可执行文件不存在，无法测试")
            return False
        
        try:
            result = subprocess.run([
                str(firefox_path), '--version'
            ], capture_output=True, text=True, timeout=10)
            
            if result.returncode == 0:
                logger.info(f"Firefox测试成功！版本: {result.stdout.strip()}")
                return True
            else:
                logger.error(f"Firefox测试失败: {result.stderr}")
                return False
                
        except subprocess.TimeoutExpired:
            logger.error("Firefox启动超时")
            return False
        except Exception as e:
            logger.error(f"测试Firefox时发生错误: {e}")
            return False


class WeixinVideoSpider:
    """
    一个用于从微信文章中抓取、下载视频并更新数据库信息的爬虫类。
    使用Playwright进行浏览器自动化，支持登录信息和公众号信息的数据库存储。
    集成浏览器管理功能，支持自动下载和管理Firefox浏览器。
    """

    def __init__(self):
        """初始化爬虫。"""
        self.db_config = DB_CONFIG
        self.base_dir = Path(__file__).resolve().parent.parent
        self.playwright = sync_playwright().start()

        # 检查并准备浏览器
        self._prepare_browser()

        # 检查数据库架构
        logger.info("检查数据库连接和架构...")
        if not self.check_database_schema():
            self.close()
            raise RuntimeError("数据库架构检查失败，请检查数据库连接配置。")

        # 获取登录凭证(优先从数据库获取)
        self.cookie_list, self.token = self._get_or_refresh_credentials()
        if not self.cookie_list or not self.token:
            self.close()
            raise RuntimeError("登录失败,未能获取cookie和token,请检查。")

        self.cookie_str = '; '.join([f"{c['name']}={c['value']}" for c in self.cookie_list])
        
        # 初始化用于爬取视频的无头浏览器
        logger.info("初始化无头浏览器用于爬取任务...")
        self.browser = self._launch_browser(headless=True)
        self.context = self.browser.new_context(
            user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        )
        self.context.add_cookies(self.cookie_list)
        logger.info("无头浏览器初始化完成。")

    def _prepare_browser(self):
        """检查并准备浏览器环境。"""
        logger.info("检查浏览器环境...")
        
        # 检查浏览器状态
        status = BrowserManager.check_browser_status()
        logger.info(f"浏览器目录: {status['browser_dir_path']}")
        logger.info(f"Firefox路径: {status['firefox_path']}")
        
        if BROWSER_CONFIG.get('use_local_browser', True):
            # 创建浏览器目录
            browser_dir = BROWSER_CONFIG['local_browser_dir']
            browser_dir.mkdir(parents=True, exist_ok=True)
            
            # 检查Firefox是否存在
            if not status['firefox_executable_exists']:
                logger.warning("本地Firefox浏览器不存在")
                if BROWSER_CONFIG.get('auto_download', True):
                    logger.info("正在自动下载Firefox浏览器...")
                    if BrowserManager.download_firefox():
                        logger.info("Firefox浏览器下载并配置成功")
                    else:
                        logger.warning("Firefox下载失败，将尝试使用系统默认浏览器")
                else:
                    logger.warning("auto_download设置为False，将使用系统默认浏览器")
            else:
                logger.info(f"找到本地Firefox浏览器: {status['firefox_path']}")
                # 测试Firefox是否正常
                if not BrowserManager.test_firefox():
                    logger.warning("Firefox测试失败，可能需要重新下载")
        else:
            logger.info("配置为使用系统默认浏览器")

    def _launch_browser(self, headless=True):
        """启动浏览器，优先使用本地浏览器。"""
        launch_options = {
            'headless': headless if BROWSER_CONFIG.get('headless') is None else BROWSER_CONFIG['headless']
        }
        
        # 添加Playwright特定配置
        if BROWSER_CONFIG.get('slow_mo'):
            launch_options['slow_mo'] = BROWSER_CONFIG['slow_mo']
        
        if BROWSER_CONFIG.get('timeout'):
            launch_options['timeout'] = BROWSER_CONFIG['timeout']
        
        # 如果配置了本地浏览器路径且文件存在
        firefox_path = BROWSER_CONFIG.get('firefox_path')
        if (BROWSER_CONFIG.get('use_local_browser', True) and 
            firefox_path and 
            Path(firefox_path).exists()):
            
            launch_options['executable_path'] = str(firefox_path)
            logger.info(f"使用本地Firefox浏览器: {firefox_path}")
        else:
            logger.info("使用系统默认Firefox浏览器")
        
        # 设置用户数据目录
        user_data_dir = BROWSER_CONFIG.get('user_data_dir')
        if user_data_dir:
            user_data_dir = Path(user_data_dir)
            user_data_dir.mkdir(parents=True, exist_ok=True)
            # 注意：Playwright的Firefox不支持user_data_dir参数
            # 这里我们只是创建目录，实际的用户数据会通过context管理
        
        try:
            browser = self.playwright.firefox.launch(**launch_options)
            logger.info("浏览器启动成功")
            return browser
        except Exception as e:
            logger.error(f"启动浏览器失败: {e}")
            # 回退到默认方式
            if 'executable_path' in launch_options:
                logger.info("回退到系统默认浏览器...")
                del launch_options['executable_path']
                try:
                    browser = self.playwright.firefox.launch(**launch_options)
                    logger.info("使用系统默认浏览器启动成功")
                    return browser
                except Exception as e2:
                    logger.error(f"系统默认浏览器启动也失败: {e2}")
                    raise
            raise

    def _get_db_connection(self):
        """获取数据库连接。"""
        try:
            return pymysql.connect(**self.db_config)
        except Exception as e:
            logger.error(f"数据库连接失败: {e}")
            return None

    def _get_or_refresh_credentials(self):
        """获取或刷新登录凭证，优先从数据库获取有效凭证。"""
        # 首先尝试从数据库获取有效凭证
        conn = self._get_db_connection()
        if conn:
            try:
                with conn.cursor() as cursor:
                    sql = """
                        SELECT cookie_value, token FROM weixin_credentials 
                        WHERE expires_at > NOW() AND status = 1
                        ORDER BY created_at DESC LIMIT 1
                    """
                    cursor.execute(sql)
                    result = cursor.fetchone()
                    if result:
                        cookie_str, token = result
                        try:
                            # 将cookie字符串转换为cookie列表
                            cookie_list = []
                            for cookie_pair in cookie_str.split('; '):
                                if '=' in cookie_pair:
                                    name, value = cookie_pair.split('=', 1)
                                    cookie_list.append({
                                        'name': name,
                                        'value': value,
                                        'domain': '.mp.weixin.qq.com',
                                        'path': '/'
                                    })
                            logger.info("从数据库成功获取到有效的登录凭证。")
                            return cookie_list, token
                        except Exception as e:
                            logger.warning(f"解析数据库中的cookie失败: {e}")
            except Exception as e:
                logger.error(f"从数据库获取凭证失败: {e}")
            finally:
                conn.close()

        # 数据库中没有有效凭证，进行扫码登录
        logger.info("数据库中无有效凭证，开始扫码登录...")
        return self._login_and_get_credentials()

    def _login_and_get_credentials(self):
        """通过启动浏览器进行扫码登录，从而获取临时的cookie和token。"""
        logger.info("启动浏览器进行扫码登录...")
        login_browser = self._launch_browser(headless=False)
        page = login_browser.new_page()
        try:
            page.goto('https://mp.weixin.qq.com', timeout=60000)
            logger.info(f"请在{WEIXIN_CONFIG['login_timeout']}秒内完成扫码登录...")
            
            # 等待URL中出现token，表明登录成功
            page.wait_for_url(re.compile(r"token=\d+"), timeout=WEIXIN_CONFIG['login_timeout'] * 1000)
            logger.info("登录成功!")

            # 获取Token
            token_match = re.search(r'token=(\d+)', page.url)
            if not token_match:
                raise ValueError("登录后无法从URL中提取token!")
            token = token_match.group(1)

            # 获取Cookie
            cookies = page.context.cookies()
            
            # 保存凭证到数据库
            self._save_credentials_to_db(cookies, token)
            
            return cookies, token
            
        except PlaywrightError as e:
            if "Timeout" in e.message:
                logger.error("登录超时，请重新运行程序。")
            else:
                logger.error(f"登录或获取凭证过程中发生Playwright错误: {e}")
            return None, None
        except Exception as e:
            logger.error(f"登录或获取凭证过程中发生未知错误: {e}")
            traceback.print_exc()
            return None, None
        finally:
            logger.info("关闭登录浏览器。")
            login_browser.close()

    def _save_credentials_to_db(self, cookies, token):
        """将获取到的cookie和token保存到数据库。"""
        conn = self._get_db_connection()
        if not conn:
            logger.error("无法连接到数据库，凭证未保存。")
            return
        
        try:
            # 先创建表（如果不存在）
            self._create_tables_if_not_exists(conn)
            
            with conn.cursor() as cursor:
                # 将旧凭证标记为无效
                cursor.execute("UPDATE weixin_credentials SET status = 0 WHERE status = 1")
                
                # 保存新凭证
                cookie_str = '; '.join([f"{c['name']}={c['value']}" for c in cookies])
                expire_time = datetime.now() + timedelta(days=WEIXIN_CONFIG['cookie_expire_days'])
                
                sql = """
                    INSERT INTO weixin_credentials (
                        cookie_value, token, expires_at, status, created_at, updated_at
                    ) VALUES (%s, %s, %s, 1, NOW(), NOW())
                """
                cursor.execute(sql, (cookie_str, token, expire_time))
            conn.commit()
            logger.info(f"已成功将新的cookie和token保存到数据库, token: {token}")
        except Exception as e:
            logger.error(f"凭证保存到数据库失败: {e}")
            conn.rollback()
        finally:
            conn.close()

    def _create_tables_if_not_exists(self, conn):
        """创建必要的数据库表（如果不存在）。"""
        tables = {
            'weixin_credentials': """
                CREATE TABLE IF NOT EXISTS weixin_credentials (
                    id INT AUTO_INCREMENT PRIMARY KEY,
                    cookie_value TEXT NOT NULL COMMENT 'Cookie字符串',
                    token VARCHAR(255) NOT NULL COMMENT '微信token',
                    expires_at DATETIME NOT NULL COMMENT '过期时间',
                    status TINYINT DEFAULT 1 COMMENT '状态: 1=有效 0=无效',
                    created_at DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
                    updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
                    INDEX idx_expires_status (expires_at, status),
                    INDEX idx_token (token)
                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='微信公众平台登录凭证表'
            """,
            
            'weixin_accounts': """
                CREATE TABLE IF NOT EXISTS weixin_accounts (
                    id INT AUTO_INCREMENT PRIMARY KEY,
                    name VARCHAR(255) NOT NULL COMMENT '公众号名称',
                    fakeid VARCHAR(255) NOT NULL COMMENT '公众号fakeid',
                    description TEXT COMMENT '公众号描述',
                    avatar_url VARCHAR(500) COMMENT '头像URL',
                    status TINYINT DEFAULT 1 COMMENT '状态: 1=有效 0=无效',
                    created_at DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
                    updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
                    UNIQUE KEY uk_name (name),
                    INDEX idx_fakeid (fakeid),
                    INDEX idx_status (status)
                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='微信公众号信息表'
            """,
            
            'hualong_third_media_backup': """
                CREATE TABLE IF NOT EXISTS hualong_third_media_backup (
                    id INT AUTO_INCREMENT PRIMARY KEY,
                    title VARCHAR(500) NOT NULL COMMENT '文章标题',
                    url VARCHAR(1000) NOT NULL COMMENT '文章URL',
                    video_url TEXT COMMENT '视频URL列表(JSON格式)',
                    spider VARCHAR(100) DEFAULT '' COMMENT '爬虫标识',
                    content TEXT COMMENT '文章内容',
                    author VARCHAR(200) COMMENT '作者',
                    publish_time DATETIME COMMENT '发布时间',
                    source VARCHAR(200) COMMENT '来源',
                    status TINYINT DEFAULT 1 COMMENT '状态: 1=有效 0=无效',
                    created_at DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
                    updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
                    INDEX idx_spider (spider),
                    INDEX idx_url (url(255)),
                    INDEX idx_status (status),
                    INDEX idx_publish_time (publish_time)
                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='华龙网第三方媒体备份表'
            """,
            
            'video_download_log': """
                CREATE TABLE IF NOT EXISTS video_download_log (
                    id INT AUTO_INCREMENT PRIMARY KEY,
                    article_id INT COMMENT '关联文章ID',
                    article_title VARCHAR(500) COMMENT '文章标题',
                    video_url VARCHAR(1000) NOT NULL COMMENT '视频URL',
                    local_path VARCHAR(1000) COMMENT '本地存储路径',
                    file_size BIGINT COMMENT '文件大小(字节)',
                    download_status TINYINT DEFAULT 0 COMMENT '下载状态: 0=待下载 1=成功 2=失败',
                    error_message TEXT COMMENT '错误信息',
                    download_time DATETIME COMMENT '下载完成时间',
                    retry_count INT DEFAULT 0 COMMENT '重试次数',
                    created_at DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
                    updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
                    INDEX idx_article_id (article_id),
                    INDEX idx_download_status (download_status),
                    INDEX idx_video_url (video_url(255))
                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='视频下载日志表'
            """,
            
            'image_download_log': """
                CREATE TABLE IF NOT EXISTS image_download_log (
                    id INT AUTO_INCREMENT PRIMARY KEY,
                    article_title VARCHAR(500) COMMENT '文章标题',
                    image_url VARCHAR(1000) COMMENT '图片URL',
                    local_path VARCHAR(1000) COMMENT '本地存储路径',
                    file_size BIGINT COMMENT '文件大小(字节)',
                    download_status TINYINT DEFAULT 0 COMMENT '下载状态: 0=待下载 1=成功 2=失败',
                    error_message TEXT COMMENT '错误信息',
                    download_time DATETIME COMMENT '下载完成时间',
                    created_at DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
                    updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
                    INDEX idx_article_title (article_title(255)),
                    INDEX idx_image_url (image_url(255)),
                    INDEX idx_download_status (download_status),
                    INDEX idx_download_time (download_time)
                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='图片下载日志表'
            """,
            
            'spider_run_log': """
                CREATE TABLE IF NOT EXISTS spider_run_log (
                    id INT AUTO_INCREMENT PRIMARY KEY,
                    run_id VARCHAR(50) NOT NULL COMMENT '运行ID',
                    start_time DATETIME NOT NULL COMMENT '开始时间',
                    end_time DATETIME COMMENT '结束时间',
                    total_articles INT DEFAULT 0 COMMENT '总文章数',
                    processed_articles INT DEFAULT 0 COMMENT '已处理文章数',
                    total_videos INT DEFAULT 0 COMMENT '总视频数',
                    downloaded_videos INT DEFAULT 0 COMMENT '已下载视频数',
                    status TINYINT DEFAULT 0 COMMENT '运行状态: 0=运行中 1=成功 2=失败',
                    error_message TEXT COMMENT '错误信息',
                    created_at DATETIME DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
                    INDEX idx_run_id (run_id),
                    INDEX idx_status (status),
                    INDEX idx_start_time (start_time)
                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='爬虫运行日志表'
            """
        }
        
        try:
            with conn.cursor() as cursor:
                logger.info("开始检查并创建数据库表...")
                
                for table_name, create_sql in tables.items():
                    try:
                        cursor.execute(create_sql)
                        logger.info(f"✓ 表 {table_name} 检查/创建成功")
                    except Exception as e:
                        logger.error(f"✗ 创建表 {table_name} 失败: {e}")
                        raise
                
                # 检查表是否存在的验证
                cursor.execute("SHOW TABLES")
                existing_tables = {row[0] for row in cursor.fetchall()}
                
                missing_tables = set(tables.keys()) - existing_tables
                if missing_tables:
                    logger.warning(f"以下表创建可能失败: {missing_tables}")
                else:
                    logger.info("所有必要的数据库表都已就绪")
                
            conn.commit()
            logger.info("数据库表创建/检查完成")
            
        except Exception as e:
            logger.error(f"创建数据库表失败: {e}")
            conn.rollback()
            raise

    def create_all_tables(self):
        """公共方法：创建所有必要的数据库表。"""
        conn = self._get_db_connection()
        if not conn:
            logger.error("无法连接到数据库，无法创建表")
            return False
        
        try:
            self._create_tables_if_not_exists(conn)
            return True
        except Exception as e:
            logger.error(f"创建表失败: {e}")
            return False
        finally:
            conn.close()

    def check_database_schema(self):
        """检查数据库架构是否完整。"""
        required_tables = [
            'weixin_credentials', 'weixin_accounts', 'hualong_third_media_backup',
            'video_download_log', 'image_download_log', 'spider_run_log'
        ]
        
        conn = self._get_db_connection()
        if not conn:
            return False
        
        try:
            with conn.cursor() as cursor:
                cursor.execute("SHOW TABLES")
                existing_tables = {row[0] for row in cursor.fetchall()}
                
                missing_tables = set(required_tables) - existing_tables
                if missing_tables:
                    logger.warning(f"缺少以下数据库表: {missing_tables}")
                    logger.info("正在自动创建缺失的表...")
                    self._create_tables_if_not_exists(conn)
                    return True
                else:
                    logger.info("数据库架构检查完成，所有表都存在")
                    return True
        except Exception as e:
            logger.error(f"检查数据库架构失败: {e}")
            return False
        finally:
            conn.close()

    def get_fakeid(self, query):
        """根据公众号名称搜索并获取其fakeid，优先从数据库获取。"""
        # 首先从数据库查找
        conn = self._get_db_connection()
        if conn:
            try:
                with conn.cursor() as cursor:
                    sql = "SELECT fakeid FROM weixin_accounts WHERE name = %s AND status = 1"
                    cursor.execute(sql, (query,))
                    result = cursor.fetchone()
                    if result:
                        fakeid = result[0]
                        logger.info(f"从数据库获取到 '{query}' 的 fakeid: {fakeid}")
                        return fakeid
            except Exception as e:
                logger.error(f"从数据库查询fakeid失败: {e}")
            finally:
                conn.close()

        # 数据库中没有，通过API获取
        logger.info(f"数据库中未找到 '{query}' 的fakeid，通过API获取...")
        return self._fetch_fakeid_from_api(query)

    def _fetch_fakeid_from_api(self, query):
        """通过微信API获取公众号fakeid。"""
        if not self.cookie_str or not self.token:
            logger.error("缺少cookie或token, 无法获取fakeid。")
            return None

        url = "https://mp.weixin.qq.com/cgi-bin/searchbiz"
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0",
            "Cookie": self.cookie_str,
            "Referer": f"https://mp.weixin.qq.com/cgi-bin/home?t=home/index&lang=zh_CN&token={self.token}"
        }
        params = {
            "action": "search_biz", "begin": 0, "count": 5, "query": query,
            "token": self.token, "lang": "zh_CN", "f": "json", "ajax": "1"
        }
        try:
            resp = requests.get(url, headers=headers, params=params, verify=False, timeout=10)
            resp.raise_for_status()
            data = resp.json()
            if data.get('base_resp', {}).get('ret') == 0:
                if data.get('list'):
                    fakeid = data['list'][0].get('fakeid')
                    logger.info(f"成功从API获取到 '{query}' 的 fakeid: {fakeid}")
                    
                    # 保存到数据库
                    self._save_fakeid_to_db(query, fakeid)
                    return fakeid
                else:
                    logger.warning(f"未找到与 '{query}' 相关的公众号。")
            else:
                err_msg = data.get('base_resp', {}).get('err_msg', '未知错误')
                logger.error(f"搜索公众号 '{query}' 失败: {err_msg}")
                if "invalid session" in err_msg or "login" in err_msg:
                    logger.error("会话可能已过期，请考虑重新运行程序。")
            return None
        except Exception as e:
            logger.error(f"请求获取 '{query}' 的fakeid时出错: {e}")
            return None

    def _save_fakeid_to_db(self, name, fakeid):
        """将公众号信息保存到数据库。"""
        conn = self._get_db_connection()
        if not conn:
            return
        
        try:
            with conn.cursor() as cursor:
                sql = """
                    INSERT INTO weixin_accounts (name, fakeid, status, created_at, updated_at)
                    VALUES (%s, %s, 1, NOW(), NOW())
                    ON DUPLICATE KEY UPDATE 
                    fakeid = VALUES(fakeid), 
                    status = 1, 
                    updated_at = NOW()
                """
                cursor.execute(sql, (name, fakeid))
            conn.commit()
            logger.info(f"已保存公众号 '{name}' 的fakeid到数据库。")
        except Exception as e:
            logger.error(f"保存fakeid到数据库失败: {e}")
            conn.rollback()
        finally:
            conn.close()

    def get_articles_with_video(self):
        """从数据库获取需要处理的文章列表。"""
        conn = self._get_db_connection()
        if not conn:
            return []
        try:
            with conn.cursor() as cursor:
                sql = "SELECT title, url, video_url FROM hualong_third_media_backup WHERE spider IS NULL OR spider = ''"
                cursor.execute(sql)
                results = cursor.fetchall()
                logger.info(f"从数据库查询到 {len(results)} 篇待处理文章")
                
                # 记录前几篇文章的信息用于调试
                for i, (title, url, video_url) in enumerate(results[:3]):
                    logger.info(f"  文章 {i+1}: {str(title)[:50]}... - {url}")
                
                return results
        except Exception as e:
            logger.error(f"查询文章失败: {e}")
            logger.error(f"错误详情: {traceback.format_exc()}")
            return []
        finally:
            if conn:
                conn.close()

    def update_video_info_in_db(self, title, video_urls):
        """更新数据库中的视频信息。"""
        conn = self._get_db_connection()
        if not conn:
            return
        try:
            with conn.cursor() as cursor:
                sql = """
                UPDATE hualong_third_media_backup 
                SET video_url = %s, spider = 'videoweixin'
                WHERE title = %s
                """
                cursor.execute(sql, (json.dumps(video_urls), title))
            conn.commit()
            logger.info(f"成功更新文章 {title} 的视频信息。")
        except Exception as e:
            logger.error(f"更新文章 {title} 的视频信息失败: {e}")
            conn.rollback()
        finally:
            if conn:
                conn.close()

    def get_video_urls(self, article_url, video_title):
        """使用Playwright获取文章中的所有视频URL。"""
        page = self.context.new_page()
        try:
            # 增加重试机制
            max_retries = 3
            for attempt in range(max_retries):
                try:
                    logger.info(f"尝试访问文章页面 (第{attempt + 1}次): {article_url}")
                    
                    # 设置更长的超时时间和更宽松的等待条件
                    page.goto(article_url, wait_until='domcontentloaded', timeout=30000)
                    
                    # 等待页面基本加载完成
                    page.wait_for_timeout(3000)  # 等待3秒让页面完全加载
                    
                    # 检查页面是否正确加载
                    title_element = page.locator('h1, .rich_media_title, #activity-name').first
                    if title_element.count() > 0:
                        logger.info("页面加载成功，开始提取视频URL")
                        break
                    else:
                        logger.warning(f"页面可能未正确加载 (第{attempt + 1}次)")
                        if attempt < max_retries - 1:
                            time.sleep(2)
                            continue
                        
                except Exception as e:
                    logger.warning(f"页面加载失败 (第{attempt + 1}次): {e}")
                    if attempt < max_retries - 1:
                        time.sleep(2)
                        continue
                    else:
                        # 最后一次尝试失败，但不抛出异常，而是尝试从页面源码提取
                        logger.warning("页面加载失败，尝试从源码提取视频")
                        break
            
            video_urls = set()
            
            try:
                page_source = page.content()
                logger.info(f"获取到页面源码，长度: {len(page_source)} 字符")
            except Exception as e:
                logger.error(f"获取页面源码失败: {e}")
                logger.error(f"错误详情: {traceback.format_exc()}")
                return []

            # 1. 处理视频号卡片
            try:
                video_cards = page.locator('.videosnap_video_iframe').all()
                logger.info(f"找到 {len(video_cards)} 个视频号卡片")
                for card in video_cards:
                    video_url = card.get_attribute('data-url')
                    if video_url:
                        video_urls.add(video_url)
                        logger.info(f"从视频号卡片获取到视频URL: {video_url[:50]}...")
                    self._save_video_channel_info(card, article_url, video_title)
            except Exception as e:
                logger.warning(f"处理视频号卡片失败: {e}")
                logger.warning(f"错误详情: {traceback.format_exc()}")

            # 2. 正则匹配直接视频链接 - 增强版本
            try:
                video_patterns = [
                    r'https://mpvideo\.qpic\.cn/[^"]*\.mp4\?[^"]*',
                    r'http://mpvideo\.qpic\.cn/[^"]*\.mp4\?[^"]*',
                    r'https://vod\.qiecdn\.com/[^"]*\.mp4\?[^"]*',
                    r'http://vod\.qiecdn\.com/[^"]*\.mp4\?[^"]*'
                ]
                for pattern in video_patterns:
                    matches = re.finditer(pattern, page_source)
                    for match in matches:
                        video_url = match.group(0)
                        video_urls.add(video_url)
                        logger.info(f"通过正则匹配找到视频URL: {video_url[:50]}...")
                
                logger.info(f"正则匹配视频链接完成，共找到 {len(video_urls)} 个")
            except Exception as e:
                logger.error(f"正则匹配视频链接失败: {e}")
                logger.error(f"错误详情: {traceback.format_exc()}")

            # 3. 通过vid获取视频链接 - 修复API调用
            try:
                vid_patterns = [
                    r"wxv_.{19}",
                    r'data-vid="([^"]+)"'
                ]
                found_vids = []
                for pattern in vid_patterns:
                    for match in re.finditer(pattern, page_source):
                        vid = match.group(1) if match.groups() else match.group(0)
                        found_vids.append(vid)
                        logger.info(f"找到视频ID: {vid}")
                
                logger.info(f"共找到 {len(found_vids)} 个视频ID")
                
                for vid in found_vids:
                    # 修复API URL构造
                    api_url = f"https://mp.weixin.qq.com/mp/videoplayer?action=get_mp_video_play_url&preview=0&__biz=MzI4MzkzMTc3OA==&mid=2247488495&idx=4&vid={vid}"
                    try:
                        headers = {
                            'Cookie': self.cookie_str,
                            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
                            'Referer': 'https://mp.weixin.qq.com/'
                        }
                        resp = requests.get(api_url, headers=headers, timeout=10)
                        resp.raise_for_status()
                        data = resp.json()
                        if data.get("url_info"):
                            for url_info in data["url_info"]:
                                if url_info.get("url"):
                                    video_urls.add(url_info.get("url"))
                                    logger.info(f"通过API获取到视频URL: {url_info.get('url')[:50]}...")
                    except Exception as e:
                        logger.warning(f"通过vid {vid} 获取视频链接失败: {e}")
            except Exception as e:
                logger.error(f"处理视频ID失败: {e}")
                logger.error(f"错误详情: {traceback.format_exc()}")

            # 4. 查找iframe中的视频
            try:
                iframes = page.locator('iframe').all()
                logger.info(f"找到 {len(iframes)} 个iframe元素")
                for iframe in iframes:
                    src = iframe.get_attribute('data-src') or iframe.get_attribute('src')
                    if src:
                        # 检查是否是视频链接
                        if ('mp.weixin.qq.com/mp/videoplayer' in src or 
                            'mpvideo.qpic.cn' in src or 
                            'vod.qiecdn.com' in src):
                            video_urls.add(src)
                            logger.info(f"从iframe获取到视频URL: {src[:50]}...")
            except Exception as e:
                logger.warning(f"处理iframe视频失败: {e}")
                logger.warning(f"错误详情: {traceback.format_exc()}")

            # 5. 查找video标签
            try:
                videos = page.locator('video').all()
                logger.info(f"找到 {len(videos)} 个video元素")
                for video in videos:
                    src = video.get_attribute('src')
                    if src:
                        video_urls.add(src)
                        logger.info(f"从video标签获取到视频URL: {src[:50]}...")
            except Exception as e:
                logger.warning(f"处理video标签失败: {e}")
                logger.warning(f"错误详情: {traceback.format_exc()}")

            # 过滤有效的视频URL
            valid_video_urls = []
            for url in video_urls:
                if url and (url.startswith('http') or url.startswith('//')):
                    if url.startswith('//'):
                        url = 'https:' + url
                    valid_video_urls.append(url)

            logger.info(f"从文章中提取到 {len(valid_video_urls)} 个视频URL")
            if valid_video_urls:
                for i, url in enumerate(valid_video_urls[:3]):
                    logger.info(f"  视频 {i+1}: {url[:80]}...")
            
            return valid_video_urls

        except Exception as e:
            logger.error(f"从文章 {article_url} 获取视频URL失败: {e}")
            logger.error(f"详细错误信息: {traceback.format_exc()}")
            return []
        finally:
            try:
                page.close()
            except Exception as e:
                logger.warning(f"关闭页面时出错: {e}")

    def _save_video_channel_info(self, card_locator, article_url, video_title):
        """保存视频号卡片信息并下载相关图片。"""
        try:
            nickname = card_locator.get_attribute('data-nickname')
            headimgurl = card_locator.get_attribute('data-headimgurl')
            desc = card_locator.get_attribute('data-desc')
            video_url = card_locator.get_attribute('data-url')
            feed_cover_url = card_locator.get_attribute('data-feedcoverurl')

            # 安全处理文件名
            safe_title = self._sanitize_filename(video_title)
            title_dir = VIDEO_CONFIG['download_dir'] / safe_title
            title_dir.mkdir(parents=True, exist_ok=True, mode=0o777)

            info_text = (
                f"视频号: {nickname}\n头像: {headimgurl}\n描述: {desc}\n"
                f"视频链接: {video_url}\n封面图: {feed_cover_url}\n"
                f"文章链接: {article_url}\n{'-'*50}\n"
            )
            
            info_file = title_dir / 'video_channel_links.txt'
            with info_file.open('a', encoding='utf-8') as f:
                f.write(info_text)

            # 下载头像和封面图
            if headimgurl:
                safe_nickname = self._sanitize_filename(nickname) if nickname else "unknown"
                avatar_path = title_dir / f'{safe_nickname}_avatar.jpg'
                self._download_image(headimgurl, avatar_path)
                
            if feed_cover_url:
                safe_nickname = self._sanitize_filename(nickname) if nickname else "unknown"
                cover_path = title_dir / f'{safe_nickname}_cover.jpg'
                self._download_image(feed_cover_url, cover_path)

        except Exception as e:
            logger.error(f"保存视频号信息失败: {e}")
            traceback.print_exc()

    def _sanitize_filename(self, filename):
        """清理文件名，移除不安全字符"""
        if not filename:
            return "unnamed"
        
        # 移除或替换不安全字符
        unsafe_chars = ['<', '>', ':', '"', '/', '\\', '|', '?', '*']
        safe_filename = filename
        for char in unsafe_chars:
            safe_filename = safe_filename.replace(char, '_')
        
        # 限制长度
        if len(safe_filename) > 100:
            safe_filename = safe_filename[:100]
            
        return safe_filename.strip()

    def _download_image(self, url, path):
        """下载图片文件，增强版本"""
        if not url:
            return False
            
        try:
            headers = {
                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
                'Referer': 'https://mp.weixin.qq.com/',
                'Accept': 'image/webp,image/apng,image/*,*/*;q=0.8'
            }
            
            response = requests.get(url, headers=headers, timeout=30, stream=True)
            response.raise_for_status()
            
            # 确保目录存在
            path.parent.mkdir(parents=True, exist_ok=True)
            
            with open(path, 'wb') as f:
                for chunk in response.iter_content(chunk_size=8192):
                    if chunk:
                        f.write(chunk)
            
            # 设置文件权限
            os.chmod(path, 0o666)
            logger.info(f"图片下载成功: {path}")
            
            # 记录图片下载到数据库
            self._log_single_image_download(str(path), url, path.stat().st_size, True)
            
            return True
            
        except Exception as e:
            logger.error(f"下载图片 {url} 失败: {e}")
            # 记录下载失败
            self._log_single_image_download(str(path) if 'path' in locals() else None, url, 0, False, str(e))
            return False

    def _log_single_image_download(self, local_path, image_url, file_size, success, error_message=None):
        """记录单个图片下载到数据库"""
        conn = self._get_db_connection()
        if not conn:
            return
            
        try:
            with conn.cursor() as cursor:
                # 从路径中提取文章标题
                article_title = "未知文章"
                if local_path:
                    try:
                        path_parts = Path(local_path).parts
                        # 查找包含文章标题的部分（通常是倒数第二个部分）
                        if len(path_parts) >= 2:
                            article_title = path_parts[-2]  # 文章标题目录
                    except Exception:
                        pass
                
                sql = """
                    INSERT INTO image_download_log 
                    (article_title, image_url, local_path, file_size, download_status, error_message, download_time)
                    VALUES (%s, %s, %s, %s, %s, %s, %s)
                """
                status = 1 if success else 2  # 1=成功, 2=失败
                download_time = datetime.now() if success else None
                
                cursor.execute(sql, (
                    article_title, image_url, local_path, file_size, 
                    status, error_message, download_time
                ))
            conn.commit()
            
        except Exception as e:
            logger.error(f"记录图片下载日志失败: {e}")
            conn.rollback()
        finally:
            conn.close()

    def download_video(self, video_url, video_title, index=0):
        """下载视频文件，增强版本"""
        if not video_url:
            logger.warning("视频URL为空，跳过下载")
            return None
            
        max_retry = VIDEO_CONFIG.get('max_retry', 3)
        
        for attempt in range(max_retry):
            try:
                # 安全处理文件名
                safe_title = self._sanitize_filename(video_title)
                title_dir = VIDEO_CONFIG['download_dir'] / safe_title
                title_dir.mkdir(parents=True, exist_ok=True, mode=0o777)
                
                video_path = title_dir / f'{safe_title}_{index}.mp4'
                
                # 如果文件已存在且大小合理，跳过下载
                if video_path.exists() and video_path.stat().st_size > 1024:
                    logger.info(f"视频文件已存在，跳过下载: {video_path}")
                    return str(video_path)

                headers = {
                    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
                    'Referer': 'https://mp.weixin.qq.com/',
                    'Accept': 'video/webm,video/ogg,video/*;q=0.9,application/ogg;q=0.7,audio/*;q=0.6,*/*;q=0.5',
                    'Accept-Encoding': 'identity',
                    'Range': 'bytes=0-'
                }
                
                logger.info(f"开始下载视频 (尝试 {attempt + 1}/{max_retry}): {video_url}")
                
                response = requests.get(
                    video_url, 
                    headers=headers, 
                    stream=True, 
                    timeout=VIDEO_CONFIG.get('timeout', 60),
                    allow_redirects=True
                )
                response.raise_for_status()
                
                # 获取文件大小
                total_size = int(response.headers.get('content-length', 0))
                
                downloaded_size = 0
                with open(video_path, 'wb') as f:
                    for chunk in response.iter_content(chunk_size=8192):
                        if chunk:
                            f.write(chunk)
                            downloaded_size += len(chunk)
                            
                            # 显示下载进度
                            if total_size > 0:
                                progress = (downloaded_size / total_size) * 100
                                if downloaded_size % (1024 * 1024) == 0:  # 每MB显示一次
                                    logger.info(f"下载进度: {progress:.1f}% ({downloaded_size}/{total_size} bytes)")
                
                # 设置文件权限
                os.chmod(video_path, 0o666)
                
                # 验证下载的文件
                if video_path.stat().st_size > 0:
                    logger.info(f"视频下载成功: {video_path} (大小: {video_path.stat().st_size} bytes)")
                    
                    # 记录下载日志到数据库
                    self._log_video_download(video_title, video_url, str(video_path), video_path.stat().st_size, True)
                    
                    return str(video_path)
                else:
                    logger.warning(f"下载的视频文件大小为0: {video_path}")
                    video_path.unlink(missing_ok=True)  # 删除空文件
                    
            except requests.exceptions.Timeout:
                logger.warning(f"视频下载超时 (尝试 {attempt + 1}/{max_retry}): {video_url}")
                if attempt < max_retry - 1:
                    time.sleep(5)  # 等待5秒后重试
                    continue
                    
            except requests.exceptions.RequestException as e:
                logger.error(f"视频下载请求失败 (尝试 {attempt + 1}/{max_retry}): {e}")
                if attempt < max_retry - 1:
                    time.sleep(5)
                    continue
                    
            except Exception as e:
                logger.error(f"下载视频失败 (尝试 {attempt + 1}/{max_retry}): {e}")
                if attempt < max_retry - 1:
                    time.sleep(5)
                    continue
        
        # 所有尝试都失败了
        logger.error(f"视频下载最终失败: {video_url}")
        self._log_video_download(video_title, video_url, None, 0, False, f"下载失败，重试{max_retry}次")
        return None

    def _log_video_download(self, article_title, video_url, local_path, file_size, success, error_message=None):
        """记录视频下载日志到数据库"""
        conn = self._get_db_connection()
        if not conn:
            return
            
        try:
            with conn.cursor() as cursor:
                sql = """
                    INSERT INTO video_download_log 
                    (article_title, video_url, local_path, file_size, download_status, error_message, download_time)
                    VALUES (%s, %s, %s, %s, %s, %s, %s)
                """
                status = 1 if success else 2  # 1=成功, 2=失败
                download_time = datetime.now() if success else None
                
                cursor.execute(sql, (
                    article_title, video_url, local_path, file_size, 
                    status, error_message, download_time
                ))
            conn.commit()
            
        except Exception as e:
            logger.error(f"记录视频下载日志失败: {e}")
            conn.rollback()
        finally:
            conn.close()

    def get_article_images(self, article_url, article_title):
        """获取文章中的所有图片URL并下载"""
        page = self.context.new_page()
        try:
            # 增加重试机制
            max_retries = 3
            for attempt in range(max_retries):
                try:
                    logger.info(f"尝试访问文章页面获取图片 (第{attempt + 1}次): {article_url}")
                    
                    # 使用更宽松的等待条件
                    page.goto(article_url, wait_until='domcontentloaded', timeout=30000)
                    
                    # 等待内容加载
                    page.wait_for_timeout(2000)  # 等待2秒
                    
                    # 检查页面是否加载成功
                    content_element = page.locator('#js_content, .rich_media_content, body').first
                    if content_element.count() > 0:
                        logger.info("页面加载成功，开始提取图片")
                        break
                    else:
                        logger.warning(f"页面内容未正确加载 (第{attempt + 1}次)")
                        if attempt < max_retries - 1:
                            time.sleep(2)
                            continue
                            
                except Exception as e:
                    logger.warning(f"页面加载失败 (第{attempt + 1}次): {e}")
                    logger.warning(f"错误详情: {traceback.format_exc()}")
                    if attempt < max_retries - 1:
                        time.sleep(2)
                        continue
                    else:
                        # 最后一次尝试失败，但不抛出异常，而是尝试从页面源码提取
                        logger.warning("页面加载失败，尝试从源码提取图片")
                        break
            
            # 获取所有图片元素
            image_urls = set()
            
            # 获取页面源码
            try:
                page_source = page.content()
                logger.info(f"获取到页面源码，长度: {len(page_source)} 字符")
            except Exception as e:
                logger.error(f"获取页面源码失败: {e}")
                logger.error(f"错误详情: {traceback.format_exc()}")
                return []
            
            # 1. 获取img标签中的图片
            try:
                img_elements = page.locator('img').all()
                logger.info(f"找到 {len(img_elements)} 个img元素")
                
                for i, img in enumerate(img_elements):
                    # 尝试获取各种可能的图片URL属性
                    src_attrs = ['data-src', 'src', 'data-original', 'data-lazy-src']
                    for attr in src_attrs:
                        try:
                            img_url = img.get_attribute(attr)
                            if img_url and self._is_valid_image_url(img_url):
                                # 处理相对URL
                                if img_url.startswith('//'):
                                    img_url = 'https:' + img_url
                                elif img_url.startswith('/'):
                                    img_url = 'https://mp.weixin.qq.com' + img_url
                                image_urls.add(img_url)
                                logger.info(f"从img标签获取到图片URL: {img_url[:50]}...")
                                break  # 找到有效URL就跳出
                        except Exception as e:
                            logger.debug(f"获取第{i+1}个img元素的{attr}属性失败: {e}")
                            continue
                
                logger.info(f"从img标签提取完成，共找到 {len(image_urls)} 张图片")
            except Exception as e:
                logger.warning(f"从img标签提取图片失败: {e}")
                logger.warning(f"错误详情: {traceback.format_exc()}")
            
            # 2. 从页面源码中使用正则表达式提取图片URL
            try:
                image_patterns = [
                    r'https://mmbiz\.qpic\.cn/[^"\']*\.(jpg|jpeg|png|gif|webp)',
                    r'https://mmbiz\.qlogo\.cn/[^"\']*\.(jpg|jpeg|png|gif|webp)',
                    r'data-src="(https://[^"]*\.(jpg|jpeg|png|gif|webp))"',
                    r'src="(https://[^"]*\.(jpg|jpeg|png|gif|webp))"'
                ]
                
                regex_image_count = 0
                for pattern in image_patterns:
                    matches = re.findall(pattern, page_source, re.IGNORECASE)
                    for match in matches:
                        img_url = match if isinstance(match, str) else match[0]
                        if self._is_valid_image_url(img_url):
                            image_urls.add(img_url)
                            regex_image_count += 1
                            logger.info(f"通过正则表达式找到图片URL: {img_url[:50]}...")
                
                logger.info(f"正则表达式提取完成，新增 {regex_image_count} 张图片")
            except Exception as e:
                logger.warning(f"从页面源码提取图片失败: {e}")
                logger.warning(f"错误详情: {traceback.format_exc()}")
            
            # 过滤并下载图片
            valid_images = list(image_urls)
            logger.info(f"文章 {article_title} 总共找到 {len(valid_images)} 张有效图片")
            
            if valid_images:
                logger.info("开始下载图片...")
                self._download_article_images(valid_images, article_title)
            else:
                logger.warning("未找到任何有效图片")
            
            return valid_images
            
        except Exception as e:
            logger.error(f"获取文章图片失败 {article_url}: {e}")
            logger.error(f"详细错误信息: {traceback.format_exc()}")
            return []
        finally:
            try:
                page.close()
            except Exception as e:
                logger.warning(f"关闭页面时出错: {e}")

    def _is_valid_image_url(self, url):
        """检查是否是有效的图片URL"""
        if not url or len(url) < 10:
            return False
        
        # 排除一些不需要的图片
        exclude_patterns = [
            'avatar',  # 头像
            'logo',    # logo
            'icon',    # 图标
            'qrcode',  # 二维码
            'placeholder'  # 占位图
        ]
        
        url_lower = url.lower()
        for pattern in exclude_patterns:
            if pattern in url_lower and ('mmbiz.qpic.cn' not in url_lower):
                return False
        
        # 检查文件扩展名
        valid_extensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp']
        return any(ext in url_lower for ext in valid_extensions)

    def _download_article_images(self, image_urls, article_title):
        """下载文章中的图片"""
        if not image_urls:
            return
        
        # 创建图片存储目录
        safe_title = self._sanitize_filename(article_title)
        images_dir = VIDEO_CONFIG['download_dir'] / safe_title / 'images'
        images_dir.mkdir(parents=True, exist_ok=True)
        
        logger.info(f"开始下载文章 {article_title} 的 {len(image_urls)} 张图片")
        
        success_count = 0
        for i, img_url in enumerate(image_urls, 1):
            try:
                # 生成文件名
                file_extension = self._get_file_extension(img_url)
                filename = f"image_{i:03d}{file_extension}"
                file_path = images_dir / filename
                
                # 如果文件已存在且大小合理，跳过
                if file_path.exists() and file_path.stat().st_size > 1024:
                    logger.info(f"图片已存在，跳过: {filename}")
                    success_count += 1
                    continue
                
                # 下载图片
                if self._download_image(img_url, file_path):
                    success_count += 1
                    logger.info(f"图片下载成功 ({i}/{len(image_urls)}): {filename}")
                else:
                    logger.warning(f"图片下载失败 ({i}/{len(image_urls)}): {img_url}")
                
                # 添加延迟避免请求过快
                time.sleep(0.5)
                
            except Exception as e:
                logger.error(f"下载图片时发生错误: {e}")
                continue
        
        logger.info(f"文章图片下载完成: {success_count}/{len(image_urls)} 张成功")

    def _get_file_extension(self, url):
        """从URL中提取文件扩展名"""
        try:
            # 从URL中提取扩展名
            url_path = url.split('?')[0]  # 去除查询参数
            if '.' in url_path:
                ext = '.' + url_path.split('.')[-1].lower()
                valid_extensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp']
                if ext in valid_extensions:
                    return ext
            return '.jpg'  # 默认扩展名
        except:
            return '.jpg'

    def _log_image_download(self, article_title, total_count, success_count):
        """记录图片下载统计到数据库"""
        conn = self._get_db_connection()
        if not conn:
            return
            
        try:
            with conn.cursor() as cursor:
                sql = """
                    INSERT INTO image_download_log 
                    (article_title, total_images, success_images, download_time)
                    VALUES (%s, %s, %s, %s)
                """
                cursor.execute(sql, (
                    article_title, total_count, success_count, datetime.now()
                ))
            conn.commit()
            
        except Exception as e:
            logger.error(f"记录图片下载统计失败: {e}")
            conn.rollback()
        finally:
            conn.close()

    def process_article_content(self, article_url, article_title):
        """处理文章内容，包括视频和图片下载"""
        logger.info(f"开始处理文章内容: {article_title}")
        
        results = {
            'title': article_title,
            'url': article_url,
            'videos': [],
            'images': [],
            'success': True,
            'error': None
        }
        
        try:
            # 1. 获取并下载视频
            logger.info("正在获取视频...")
            video_urls = self.get_video_urls(article_url, article_title)
            if video_urls:
                logger.info(f"找到 {len(video_urls)} 个视频")
                self.update_video_info_in_db(article_title, video_urls)
                for i, video_url in enumerate(video_urls):
                    video_path = self.download_video(video_url, article_title, i)
                    if video_path:
                        results['videos'].append({
                            'url': video_url,
                            'local_path': video_path,
                            'index': i
                        })
            
            # 2. 获取并下载图片
            logger.info("正在获取图片...")
            image_urls = self.get_article_images(article_url, article_title)
            results['images'] = image_urls
            
            # 3. 添加处理延迟
            time.sleep(random.uniform(
                WEIXIN_CONFIG.get('request_delay_min', 2), 
                WEIXIN_CONFIG.get('request_delay_max', 5)
            ))
            
        except Exception as e:
            logger.error(f"处理文章内容时发生错误: {e}")
            results['success'] = False
            results['error'] = str(e)
            traceback.print_exc()
        
        return results

    def get_history_articles(self, fakeid, account_name, offset=0, count=10):
        """获取公众号历史文章列表 - 基于参考代码实现"""
        if not self.token or not self.cookie_str:
            logger.error("缺少微信凭证，无法获取文章")
            return []
        
        url = "https://mp.weixin.qq.com/cgi-bin/appmsgpublish"
        headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0",
            "Accept": "*/*", 
            "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6",
            "Accept-Encoding": "gzip, deflate, br, zstd",
            "X-Requested-With": "XMLHttpRequest",
            "Cache-Control": "no-cache, must-revalidate",
            "Cookie": self.cookie_str
        }
        
        params = {
            "sub": "list",
            "search_field": "null",
            "begin": str(offset),
            "count": str(count),
            "query": "",
            "fakeid": fakeid,
            "type": "101_1", 
            "free_publish_type": "1",
            "sub_action": "list_ex",
            "lang": "zh_CN",
            "f": "json",
            "ajax": "1",
            "token": self.token
        }
        
        try:
            logger.info(f"正在获取 {account_name} 的文章列表...")
            resp = requests.get(url, headers=headers, params=params, verify=False, timeout=30)
            
            if resp.status_code == 200:
                # 记录响应内容用于调试
                logger.debug(f"API响应状态码: {resp.status_code}")
                logger.debug(f"响应内容前500字符: {resp.text[:500]}")
                
                try:
                    resp_json = resp.json()
                except json.JSONDecodeError as e:
                    logger.error(f"JSON解析失败: {e}")
                    logger.error(f"响应内容: {resp.text}")
                    return []
                
                # 检查响应结构
                if not isinstance(resp_json, dict):
                    logger.error(f"响应不是字典格式: {type(resp_json)}")
                    return []
                
                base_resp = resp_json.get('base_resp', {})
                if not isinstance(base_resp, dict):
                    logger.error(f"base_resp不是字典格式: {type(base_resp)}")
                    return []
                
                if base_resp.get('err_msg') == 'ok':
                    publish_page_str = resp_json.get('publish_page', '{}')
                    if isinstance(publish_page_str, str):
                        try:
                            publish_page = json.loads(publish_page_str)
                        except json.JSONDecodeError:
                            logger.error(f"publish_page JSON解析失败: {publish_page_str}")
                            return []
                    else:
                        publish_page = publish_page_str
                    
                    if not isinstance(publish_page, dict):
                        logger.error(f"publish_page不是字典格式: {type(publish_page)}")
                        return []
                    
                    articles = []
                    publish_list = publish_page.get('publish_list', [])
                    
                    for item in publish_list:
                        if not isinstance(item, dict):
                            continue
                            
                        publish_info_str = item.get('publish_info', '{}')
                        if isinstance(publish_info_str, str):
                            try:
                                publish_info = json.loads(publish_info_str)
                            except json.JSONDecodeError:
                                logger.warning(f"publish_info JSON解析失败，跳过该条目")
                                continue
                        else:
                            publish_info = publish_info_str
                        
                        if not isinstance(publish_info, dict):
                            continue
                        
                        # 获取发布时间
                        sent_info = publish_info.get('sent_info', {})
                        publish_time = sent_info.get('time')
                        if not publish_time:
                            logger.debug("缺少发布时间，跳过该文章")
                            continue
                        
                        # 获取文章列表
                        appmsgex = publish_info.get('appmsgex', [])
                        for article in appmsgex:
                            if not isinstance(article, dict):
                                continue
                                
                            article_data = {
                                'title': article.get('title', '无标题'),
                                'url': article.get('link', ''),
                                'author': article.get('author', account_name),
                                'digest': article.get('digest', ''),
                                'cover': article.get('cover', ''),
                                'publish_time': publish_time,
                                'account_name': account_name
                            }
                            
                            if article_data['url']:  # 只添加有URL的文章
                                articles.append(article_data)
                    
                    logger.info(f"成功获取 {account_name} 的 {len(articles)} 篇文章")
                    return articles
                else:
                    error_msg = base_resp.get('err_msg', '未知错误')
                    logger.error(f"获取 {account_name} 文章失败: {error_msg}")
                    
                    # 检查是否是登录相关错误
                    if 'login' in error_msg.lower() or 'invalid' in error_msg.lower():
                        logger.error("可能是登录凭证过期，请重新获取微信凭证")
                    
                    return []
            else:
                logger.error(f"请求失败，状态码: {resp.status_code}")
                logger.error(f"响应内容: {resp.text}")
                return []
                
        except Exception as e:
            logger.error(f"获取 {account_name} 文章时出错: {e}")
            import traceback
            logger.error(f"详细错误信息: {traceback.format_exc()}")
            return []

    def get_article_full_content(self, article_url):
        """获取文章完整内容"""
        page = self.context.new_page()
        try:
            logger.info(f"正在获取文章内容: {article_url}")
            
            # 访问文章页面
            page.goto(article_url, wait_until='domcontentloaded', timeout=30000)
            page.wait_for_timeout(3000)  # 等待页面加载完成
            
            # 提取文章内容 - 尝试多种选择器
            content = ""
            content_selectors = [
                "#js_content",           # 微信文章主要内容区域
                ".rich_media_content",   # 富媒体内容
                ".weui-msg__text-area",  # 微信消息文本区域
                "[id*='content']",       # 包含content的id
                ".msg_content",          # 消息内容
                "article",               # 文章标签
                ".article-content"       # 文章内容类
            ]
            
            for selector in content_selectors:
                try:
                    content_element = page.locator(selector).first
                    if content_element.count() > 0:
                        text_content = content_element.inner_text()
                        if text_content and len(text_content.strip()) > 100:  # 确保内容不为空且有实质内容
                            content = text_content.strip()
                            logger.info(f"通过选择器 {selector} 获取到内容，长度: {len(content)} 字符")
                            break
                except Exception as e:
                    logger.debug(f"选择器 {selector} 提取失败: {e}")
                    continue
            
            # 如果没有获取到内容，尝试从页面源码中提取
            if not content or len(content) < 100:
                try:
                    page_source = page.content()
                    from bs4 import BeautifulSoup
                    soup = BeautifulSoup(page_source, 'html.parser')
                    
                    # 移除不需要的标签
                    for tag in soup.find_all(['script', 'style', 'nav', 'header', 'footer']):
                        tag.decompose()
                    
                    # 查找可能的内容容器
                    content_containers = [
                        soup.find('div', {'id': 'js_content'}),
                        soup.find('div', class_=lambda x: x and 'content' in x.lower()),
                        soup.find('article'),
                        soup.find('div', class_=lambda x: x and 'article' in x.lower())
                    ]
                    
                    for container in content_containers:
                        if container:
                            # 获取所有段落文本
                            paragraphs = container.find_all(['p', 'div'])
                            texts = [p.get_text().strip() for p in paragraphs if p.get_text().strip() and len(p.get_text().strip()) > 20]
                            if texts:
                                content = '\n'.join(texts)
                                logger.info(f"从HTML源码提取到内容，长度: {len(content)} 字符")
                                break
                    
                    # 如果还是没有内容，获取所有段落
                    if not content:
                        paragraphs = soup.find_all('p')
                        texts = [p.get_text().strip() for p in paragraphs if len(p.get_text().strip()) > 50]
                        if texts:
                            content = '\n'.join(texts)
                            logger.info(f"从所有段落提取到内容，长度: {len(content)} 字符")
                
                except Exception as e:
                    logger.warning(f"从HTML源码提取内容失败: {e}")
            
            if content and len(content) > 100:
                logger.info(f"成功获取文章内容，长度: {len(content)} 字符")
                return content
            else:
                logger.warning(f"未能获取到有效的文章内容，内容长度: {len(content) if content else 0}")
                return ""
                
        except Exception as e:
            logger.error(f"获取文章内容失败 {article_url}: {e}")
            logger.error(f"详细错误信息: {traceback.format_exc()}")
            return ""
        finally:
            try:
                page.close()
            except Exception as e:
                logger.warning(f"关闭页面时出错: {e}")

    def save_articles_to_db(self, articles):
        """保存文章到数据库，包含完整内容"""
        if not articles:
            return 0
        
        conn = self._get_db_connection()
        if not conn:
            return 0
        
        saved_count = 0
        try:
            with conn.cursor() as cursor:
                for article in articles:
                    # 检查文章是否已存在
                    check_sql = "SELECT COUNT(*) FROM hualong_third_media_backup WHERE url = %s"
                    cursor.execute(check_sql, (article['url'],))
                    if cursor.fetchone()[0] > 0:
                        logger.info(f"文章已存在，跳过: {article['title'][:50]}...")
                        continue
                    
                    # 获取文章完整内容
                    logger.info(f"正在获取文章完整内容: {article['title'][:50]}...")
                    full_content = self.get_article_full_content(article['url'])
                    
                    # 如果没有获取到完整内容，使用摘要作为备选
                    if not full_content:
                        full_content = article.get('digest', '')
                        logger.warning(f"未获取到完整内容，使用摘要: {len(full_content)} 字符")
                    
                    # 插入新文章
                    insert_sql = """
                    INSERT INTO hualong_third_media_backup 
                    (title, url, content, spider, author, source, publish_time, status, created_at, updated_at)
                    VALUES (%s, %s, %s, %s, %s, %s, %s, 1, NOW(), NOW())
                    """
                    
                    # 转换时间戳为datetime对象
                    from datetime import datetime
                    publish_time = None
                    if article.get('publish_time'):
                        try:
                            publish_time = datetime.fromtimestamp(int(article['publish_time']))
                        except (ValueError, TypeError):
                            logger.warning(f"无效的时间戳: {article.get('publish_time')}")
                    
                    cursor.execute(insert_sql, (
                        article['title'],
                        article['url'],
                        full_content,  # 使用完整内容而不是摘要
                        '',  # spider字段为空，符合处理条件
                        article['author'],
                        article['account_name'],
                        publish_time
                    ))
                    saved_count += 1
                    logger.info(f"保存文章成功: {article['title'][:50]}... (内容长度: {len(full_content)} 字符)")
                    
                    # 添加延迟避免请求过快
                    time.sleep(random.uniform(1, 3))
            
            conn.commit()
            logger.info(f"成功保存 {saved_count} 篇文章到数据库，包含完整内容")
            
        except Exception as e:
            logger.error(f"保存文章到数据库失败: {e}")
            logger.error(f"详细错误信息: {traceback.format_exc()}")
            conn.rollback()
        finally:
            conn.close()
        
        return saved_count

    def fetch_real_articles(self, articles_per_account=10):
        """获取真实的公众号文章"""
        # 目标公众号信息 - 来自参考代码
        target_accounts = {
            "重庆发布": "MzA4MzA0MTA4NA==",
            "重庆日报": "MjM5MjI5MDI1NA==", 
            "华龙网": "MjM5MzkwODc5MQ=="
        }
        
        logger.info("开始获取真实的公众号文章...")
        total_saved = 0
        
        for account_name, fakeid in target_accounts.items():
            try:
                logger.info(f"\n{'='*50}")
                logger.info(f"正在处理公众号: {account_name}")
                
                # 获取文章列表
                articles = self.get_history_articles(fakeid, account_name, count=articles_per_account)
                
                if articles:
                    # 保存到数据库
                    saved_count = self.save_articles_to_db(articles)
                    total_saved += saved_count
                    logger.info(f"{account_name} 完成，保存了 {saved_count} 篇文章")
                else:
                    logger.warning(f"{account_name} 未获取到文章")
                
                # 避免请求过于频繁
                time.sleep(random.uniform(2, 5))
                
            except Exception as e:
                logger.error(f"处理公众号 {account_name} 时出错: {e}")
                continue
        
        logger.info(f"\n{'='*50}")
        logger.info(f"文章获取完成！总共保存了 {total_saved} 篇真实文章")
        return total_saved

    def run(self):
        """爬虫主运行逻辑 - 修改为先获取真实文章再处理视频"""
        logger.info("开始运行微信公众号爬虫...")
        
        # 1. 首先获取真实的文章
        logger.info("步骤1: 获取真实的公众号文章...")
        fetched_count = self.fetch_real_articles(articles_per_account=15)
        
        if fetched_count == 0:
            logger.warning("未获取到任何真实文章，可能是凭证问题")
            logger.info("尝试处理数据库中现有的文章...")
        
        # 2. 处理文章中的视频和图片
        logger.info("步骤2: 处理文章中的视频和图片...")
        articles = self.get_articles_with_video()
        logger.info(f"共找到 {len(articles)} 篇待处理的文章。")

        processed_count = 0
        for title, url, _ in articles:
            title = str(title)
            logger.info(f"正在处理文章: {title[:50]}... - {url}")
            try:
                results = self.process_article_content(url, title)
                if results['success']:
                    logger.info(f"文章 {title[:30]}... 处理成功")
                    processed_count += 1
                else:
                    logger.error(f"文章 {title[:30]}... 处理失败: {results['error']}")
                
                time.sleep(random.uniform(WEIXIN_CONFIG['request_delay_min'], WEIXIN_CONFIG['request_delay_max']))

            except Exception as e:
                logger.error(f"处理文章 {title[:30]}... 时发生未知错误: {e}")
                traceback.print_exc()
                continue
        
        logger.info(f"爬虫运行完成！获取了 {fetched_count} 篇新文章，处理了 {processed_count} 篇文章的媒体内容")

    def close(self):
        """关闭浏览器驱动，释放资源。"""
        if hasattr(self, 'browser') and self.browser:
            self.browser.close()
        if hasattr(self, 'playwright') and self.playwright:
            self.playwright.stop()
        logger.info("Playwright资源已关闭。")


if __name__ == "__main__":
    spider = None
    try:
        spider = WeixinVideoSpider()

        # --- 新增功能演示: 获取公众号fakeid ---
        logger.info("="*20 + " 开始获取公众号 fakeid " + "="*20)
        queries = ["重庆发布", "重庆日报", "华龙网"]
        fakeids = {}
        for query in queries:
            fakeid = spider.get_fakeid(query)
            if fakeid:
                fakeids[query] = fakeid
            time.sleep(random.uniform(WEIXIN_CONFIG['request_delay_min'], WEIXIN_CONFIG['request_delay_max']))
        
        logger.info(f"获取到的Fakeid列表: {json.dumps(fakeids, indent=4, ensure_ascii=False)}")
        logger.info("="*20 + " fakeid获取结束 " + "="*20)
        
        # --- 原有功能: 抓取文章视频 ---
        logger.info("\n" + "="*20 + " 开始抓取文章视频 " + "="*20)
        spider.run()

    except Exception as e:
        logger.critical(f"爬虫启动或运行时发生严重错误: {e}")
        traceback.print_exc()
    finally:
        if spider:
            spider.close() 